[1:] movl reg=0; \
.xdata4 ".data.patch.running_on_xen", 1b-.
- .section ".data.patch.brl_xen_rsm_be_i", "a"
- .previous
-#define BRL_COND_XEN_RSM_BE_I(pr) \
-[1:](pr)brl.cond.sptk 0; \
- .xdata4 ".data.patch.brl_xen_rsm_be_i", 1b-.
-
- .section ".data.patch.brl_xen_get_psr", "a"
- .previous
-#define BRL_COND_XEN_GET_PSR(pr) \
-[1:](pr)brl.cond.sptk 0; \
- .xdata4 ".data.patch.brl_xen_get_psr", 1b-.
-
.section ".data.patch.brl_xen_ssm_i_0", "a"
.previous
#define BRL_COND_XEN_SSM_I_0(pr) \
cmp.ne isXen,isRaw=r0,r30
;;
(isRaw) rsm psr.be | psr.i
- BRL_COND_XEN_RSM_BE_I(isXen)
- .global .vdso_rsm_be_i_ret
-.vdso_rsm_be_i_ret:
+(isXen) st1 [r22]=r20
+(isXen) XEN_HYPER_RSM_BE
+ ;;
#else
rsm psr.be | psr.i // M2 (5 cyc to srlz.d)
#endif
lfetch [r18] // M0|1
#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
(isRaw) mov r29=psr
- BRL_COND_XEN_GET_PSR(isXen)
- .global .vdso_get_psr_ret
-.vdso_get_psr_ret:
+(isXen) XEN_HYPER_GET_PSR
+ ;;
+(isXen) mov r29=r8
#else
mov r29=psr // M2 (12 cyc)
#endif
*(.data.patch.running_on_xen)
__end_gate_running_on_xen_patchlist = .;
- __start_gate_brl_xen_rsm_be_i_patchlist = .;
- *(.data.patch.brl_xen_rsm_be_i)
- __end_gate_brl_xen_rsm_be_i_patchlist = .;
-
- __start_gate_brl_xen_get_psr_patchlist = .;
- *(.data.patch.brl_xen_get_psr)
- __end_gate_brl_xen_get_psr_patchlist = .;
-
__start_gate_brl_xen_ssm_i_0_patchlist = .;
*(.data.patch.brl_xen_ssm_i_0)
__end_gate_brl_xen_ssm_i_0_patchlist = .;
static void
patch_brl_in_vdso(void)
{
- EXTERN_PATCHLIST(xen_rsm_be_i);
- EXTERN_PATCHLIST(xen_get_psr);
EXTERN_PATCHLIST(xen_ssm_i_0);
EXTERN_PATCHLIST(xen_ssm_i_1);
- PATCH_BRL_SYMADDR(xen_rsm_be_i);
- PATCH_BRL_SYMADDR(xen_get_psr);
PATCH_BRL_SYMADDR(xen_ssm_i_0);
PATCH_BRL_SYMADDR(xen_ssm_i_1);
}
#include <asm/processor.h>
#include <asm/asmmacro.h>
-/* To clear vpsr.ic, vpsr.i needs to be cleared first */
-#define XEN_CLEAR_PSR_IC \
- mov r14=1; \
- movl r15=XSI_PSR_I_ADDR; \
- movl r2=XSI_PSR_IC; \
- ;; \
- ld8 r15=[r15]; \
- ld4 r3=[r2]; \
- ;; \
- ld1 r16=[r15]; \
- ;; \
- st1 [r15]=r14; \
- st4 [r2]=r0; \
- ;;
-
-/* First restore vpsr.ic, and then vpsr.i */
-#define XEN_RESTORE_PSR_IC \
- st4 [r2]=r3; \
- st1 [r15]=r16; \
- ;;
-
GLOBAL_ENTRY(xen_get_ivr)
movl r8=running_on_xen;;
ld4 r8=[r8];;
cmp.eq p7,p0=r8,r0;;
(p7) mov r8=cr.ivr;;
(p7) br.ret.sptk.many rp
- ;;
- XEN_CLEAR_PSR_IC
;;
XEN_HYPER_GET_IVR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_get_ivr)
cmp.eq p7,p0=r8,r0;;
(p7) mov r8=cr.tpr;;
(p7) br.ret.sptk.many rp
- ;;
- XEN_CLEAR_PSR_IC
;;
XEN_HYPER_GET_TPR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_get_tpr)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_SET_TPR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_set_tpr)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_EOI
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_eoi)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_THASH
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_thash)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_SET_ITM
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_set_itm)
mov r8=r32
mov r9=r33
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_PTC_GA
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_ptcga)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_GET_RR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_get_rr)
mov r8=r32
mov r9=r33
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_SET_RR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
;;
END(xen_set_rr)
1: mov r8=r32
mov r9=r33
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_SET_KR
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_set_kr)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_FC
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_fc)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_GET_CPUID
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_get_cpuid)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_GET_PMD
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_get_pmd)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_GET_EFLAG
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_get_eflag)
;;
mov r8=r32
;;
- XEN_CLEAR_PSR_IC
- ;;
XEN_HYPER_SET_EFLAG
;;
- XEN_RESTORE_PSR_IC
- ;;
br.ret.sptk.many rp
END(xen_set_eflag)
#endif
#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
// Those are vdso specialized.
// In fsys mode, call, ret can't be used.
-GLOBAL_ENTRY(xen_rsm_be_i)
- st1 [r22]=r20
- st4 [r23]=r0
- XEN_HYPER_RSM_BE
- st4 [r23]=r20
- brl.cond.sptk .vdso_rsm_be_i_ret
- ;;
-END(xen_rsm_be_i)
-
-GLOBAL_ENTRY(xen_get_psr)
- mov r31=r8
- mov r25=IA64_PSR_IC
- st4 [r23]=r0
- XEN_HYPER_GET_PSR
- ;;
- st4 [r23]=r20
- or r29=r8,r25 // vpsr.ic was cleared for hyperprivop
- mov r8=r31
- brl.cond.sptk .vdso_get_psr_ret
- ;;
-END(xen_get_psr)
// see xen_ssm_i() in privop.h
// r22 = &vcpu->vcpu_info->evtchn_upcall_mask
(p14) cmp.ne.unc p11,p0=r0,r25; \
;; \
(p11) st1 [r22]=r20; \
-(p11) st4 [r23]=r0; \
(p11) XEN_HYPER_SSM_I;
GLOBAL_ENTRY(xen_ssm_i_0)
shr.u r26=r20,IA64_GRANULE_SHIFT
cmp.eq p7,p6=r25,in0
;;
-#ifdef CONFIG_XEN
- movl r8=XSI_PSR_IC
- ;;
- st4 [r8]=r0 // force psr.ic off for hyperprivop(s)
- ;;
-#endif
/*
* If we've already mapped this task's page, we can skip doing it again.
*/
(p6) br.cond.dpnt .map
;;
.done:
+ ld8 sp=[r21] // load kernel stack pointer of new task
#ifdef CONFIG_XEN
- // psr.ic already off
// update "current" application register
mov r8=IA64_KR_CURRENT
mov r9=in0;;
XEN_HYPER_SET_KR
- ld8 sp=[r21] // load kernel stack pointer of new task
- movl r27=XSI_PSR_IC
- mov r8=1
- ;;
- st4 [r27]=r8 // psr.ic back on
#else
- ld8 sp=[r21] // load kernel stack pointer of new task
mov IA64_KR(CURRENT)=in0 // update "current" application register
#endif
mov r8=r13 // return pointer to previously running task
.map:
#ifdef CONFIG_XEN
- // psr.ic already off
+ movl r25=XSI_PSR_IC // clear psr.ic
+ ;;
+ st4 [r25]=r0
+ ;;
#else
rsm psr.ic // interrupts (psr.i) are already disabled here
#endif
#endif
;;
itr.d dtr[r25]=r23 // wire in new mapping...
-#ifndef CONFIG_XEN
+#ifdef CONFIG_XEN
+ mov r9=1
+ movl r8=XSI_PSR_IC
+ ;;
+ st4 [r8]=r9
+ ;;
+#else
ssm psr.ic // reenable the psr.ic bit
;;
srlz.d
;;
#endif
#ifdef CONFIG_XEN
- br.cond.sptk.many xen_page_fault
+
+#define MASK_TO_PEND_OFS (-1)
+
+(p15) movl r14=XSI_PSR_I_ADDR
;;
-done_xen_page_fault:
-#endif
+(p15) ld8 r14=[r14]
+ ;;
+(p15) st1 [r14]=r0,MASK_TO_PEND_OFS // if (p15) vpsr.i = 1
+ ;; // if (p15) (vcpu->vcpu_info->evtchn_upcall_mask)=0
+(p15) ld1 r14=[r14] // if (vcpu->vcpu_info->evtchn_upcall_pending)
+ ;;
+(p15) cmp.ne p15,p0=r14,r0
+ ;;
+(p15) XEN_HYPER_SSM_I
+#else
(p15) ssm psr.i // restore psr.i
+#endif
movl r14=ia64_leave_kernel
;;
SAVE_REST
ENTRY(dkey_miss)
DBG_FAULT(7)
FAULT(7)
-#ifdef CONFIG_XEN
- // Leaving this code inline above results in an IVT section overflow
- // There is no particular reason for this code to be here...
-xen_page_fault:
-(p15) movl r3=XSI_PSR_I_ADDR
- ;;
-(p15) ld8 r3=[r3]
- ;;
-(p15) st1 [r3]=r0,-1 // if (p15) vpsr.i = 1
- mov r14=r0
- ;;
-(p15) ld1 r14=[r3] // if (pending_events)
- adds r3=8,r2 // re-set up second base pointer
- ;;
-(p15) cmp.ne p15,p0=r14,r0
- ;;
- br.cond.sptk.many done_xen_page_fault
- ;;
-#endif
END(dkey_miss)
.org ia64_ivt+0x2000
#ifdef CONFIG_XEN
(p15) ld8 r16=[r16] // vpsr.i
;;
-(p15) st1 [r16]=r0,-1 // if (p15) vpsr.i = 1
- mov r2=r0
- ;;
-(p15) ld1 r2=[r16] // if (pending_events)
+(p15) st1 [r16]=r0,MASK_TO_PEND_OFS // if (p15) vpsr.i = 1
+ ;; // if (p15) (vcpu->vcpu_info->evtchn_upcall_mask)=0
+(p15) ld1 r2=[r16] // if (vcpu->vcpu_info->evtchn_upcall_pending)
;;
- cmp.ne p6,p0=r2,r0
+(p15) cmp.ne.unc p6,p0=r2,r0
;;
-(p6) ssm psr.i // do a real ssm psr.i
+(p6) XEN_HYPER_SSM_I // do a real ssm psr.i
#else
(p15) ssm psr.i // M2 restore psr.i
#endif
#ifdef CONFIG_XEN
// this is low priority for paravirtualization, but is called
// from the idle loop so confuses privop counting
- movl r31=XSI_PSR_IC
+ movl r31=XSI_PSR_I_ADDR
;;
-(p6) st4 [r31]=r0
+ ld8 r31=[r31]
+ mov r22=1
;;
-(p7) adds r31=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r31
-(p7) mov r22=1
+ st1 [r31]=r22
+ ;;
+(p6) movl r31=XSI_PSR_IC
;;
-(p7) ld8 r31=[r31]
- ;;
-(p7) st1 [r31]=r22
+(p6) st4.rel [r31]=r0
;;
mov r31 = in3
mov b7 = loc2
* that we inline it */
#define xen_hyper_ssm_i() \
({ \
- xen_set_virtual_psr_i(0); \
- xen_set_virtual_psr_ic(0); \
XEN_HYPER_SSM_I; \
})
#define xen_ssm_i() \
({ \
int old = xen_get_virtual_psr_i(); \
- xen_set_virtual_psr_i(1); \
- if (!old && xen_get_virtual_pend()) xen_hyper_ssm_i(); \
+ if (!old) { \
+ if (xen_get_virtual_pend()) \
+ xen_hyper_ssm_i(); \
+ else \
+ xen_set_virtual_psr_i(1); \
+ } \
})
#define xen_ia64_intrin_local_irq_restore(x) \
vcpu_increment_iip(current);
} else
reflect_interruption(isr, regs, vector);
- } else if (!PSCB(v, interrupt_collection_enabled)) {
+ } else if ((iim - HYPERPRIVOP_START) < HYPERPRIVOP_MAX
+ && ia64_get_cpl(regs->cr_ipsr) == 2) {
if (ia64_hyperprivop(iim, regs))
vcpu_increment_iip(current);
} else {
#ifdef XEN
#include <asm/debugger.h>
#include <asm/vhpt.h>
+#include <public/arch-ia64.h>
#endif
/*
* arch/ia64/kernel/ivt.S
#endif
movl r18=THIS_CPU(current_psr_ic_addr)
;;
- ld8 r18=[r18]
- ;;
- ld4 r19=[r18]
+ ld8 r18=[r18]
;;
cmp.eq p7,p0=r0,r17 // is this a psuedo-cover?
(p7) br.spnt.many dispatch_privop_fault
;;
- // If vpsr.ic is off, we have a hyperprivop. A hyperprivop is
- // hand-coded assembly with psr.ic off which means it can make
- // no calls, cannot use r1-r15, and it can have no memory accesses
- // unless they are to pinned addresses!
- cmp4.eq p7,p0=r0,r19
+ // if (ipsr.cpl == 2 && (iim - HYPERPRIVOP_START) < HYPERPRIVOP_MAX)
+ // this is a hyperprivop. A hyperprivop is hand-coded assembly with
+ // psr.ic off which means it can make no calls, cannot use r1-r15,
+ // and it can have no memory accesses unless they are to pinned
+ // addresses!
+ mov r19= cr.ipsr
+ movl r20=HYPERPRIVOP_START
+ mov r21=HYPERPRIVOP_MAX
+ ;;
+ sub r20=r17,r20
+ extr.u r19=r19,IA64_PSR_CPL0_BIT,2 // extract cpl field from cr.ipsr
+ ;;
+ cmp.gtu p7,p0=r21,r20
+ ;;
+ cmp.eq.and p7,p0=2,r19 // ipsr.cpl==2
(p7) br.sptk.many fast_hyperprivop
;;
movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET
#define XMAPPEDREGS_OFS XSI_SIZE
/* Hyperprivops. */
-#define HYPERPRIVOP_RFI 0x1
-#define HYPERPRIVOP_RSM_DT 0x2
-#define HYPERPRIVOP_SSM_DT 0x3
-#define HYPERPRIVOP_COVER 0x4
-#define HYPERPRIVOP_ITC_D 0x5
-#define HYPERPRIVOP_ITC_I 0x6
-#define HYPERPRIVOP_SSM_I 0x7
-#define HYPERPRIVOP_GET_IVR 0x8
-#define HYPERPRIVOP_GET_TPR 0x9
-#define HYPERPRIVOP_SET_TPR 0xa
-#define HYPERPRIVOP_EOI 0xb
-#define HYPERPRIVOP_SET_ITM 0xc
-#define HYPERPRIVOP_THASH 0xd
-#define HYPERPRIVOP_PTC_GA 0xe
-#define HYPERPRIVOP_ITR_D 0xf
-#define HYPERPRIVOP_GET_RR 0x10
-#define HYPERPRIVOP_SET_RR 0x11
-#define HYPERPRIVOP_SET_KR 0x12
-#define HYPERPRIVOP_FC 0x13
-#define HYPERPRIVOP_GET_CPUID 0x14
-#define HYPERPRIVOP_GET_PMD 0x15
-#define HYPERPRIVOP_GET_EFLAG 0x16
-#define HYPERPRIVOP_SET_EFLAG 0x17
-#define HYPERPRIVOP_RSM_BE 0x18
-#define HYPERPRIVOP_GET_PSR 0x19
-#define HYPERPRIVOP_MAX 0x19
+#define HYPERPRIVOP_START 0x1
+#define HYPERPRIVOP_RFI (HYPERPRIVOP_START + 0x0)
+#define HYPERPRIVOP_RSM_DT (HYPERPRIVOP_START + 0x1)
+#define HYPERPRIVOP_SSM_DT (HYPERPRIVOP_START + 0x2)
+#define HYPERPRIVOP_COVER (HYPERPRIVOP_START + 0x3)
+#define HYPERPRIVOP_ITC_D (HYPERPRIVOP_START + 0x4)
+#define HYPERPRIVOP_ITC_I (HYPERPRIVOP_START + 0x5)
+#define HYPERPRIVOP_SSM_I (HYPERPRIVOP_START + 0x6)
+#define HYPERPRIVOP_GET_IVR (HYPERPRIVOP_START + 0x7)
+#define HYPERPRIVOP_GET_TPR (HYPERPRIVOP_START + 0x8)
+#define HYPERPRIVOP_SET_TPR (HYPERPRIVOP_START + 0x9)
+#define HYPERPRIVOP_EOI (HYPERPRIVOP_START + 0xa)
+#define HYPERPRIVOP_SET_ITM (HYPERPRIVOP_START + 0xb)
+#define HYPERPRIVOP_THASH (HYPERPRIVOP_START + 0xc)
+#define HYPERPRIVOP_PTC_GA (HYPERPRIVOP_START + 0xd)
+#define HYPERPRIVOP_ITR_D (HYPERPRIVOP_START + 0xe)
+#define HYPERPRIVOP_GET_RR (HYPERPRIVOP_START + 0xf)
+#define HYPERPRIVOP_SET_RR (HYPERPRIVOP_START + 0x10)
+#define HYPERPRIVOP_SET_KR (HYPERPRIVOP_START + 0x11)
+#define HYPERPRIVOP_FC (HYPERPRIVOP_START + 0x12)
+#define HYPERPRIVOP_GET_CPUID (HYPERPRIVOP_START + 0x13)
+#define HYPERPRIVOP_GET_PMD (HYPERPRIVOP_START + 0x14)
+#define HYPERPRIVOP_GET_EFLAG (HYPERPRIVOP_START + 0x15)
+#define HYPERPRIVOP_SET_EFLAG (HYPERPRIVOP_START + 0x16)
+#define HYPERPRIVOP_RSM_BE (HYPERPRIVOP_START + 0x17)
+#define HYPERPRIVOP_GET_PSR (HYPERPRIVOP_START + 0x18)
+#define HYPERPRIVOP_MAX (0x19)
/* Fast and light hypercalls. */
#define __HYPERVISOR_ia64_fast_eoi 0x0200
projects / xen.git / commitdiff